import intake
import ast
from distributed import Client
from ncar_jobqueue import NCARCluster
cluster = NCARCluster()
cluster.scale(10)
client = Client(cluster)
/glade/u/home/mgrover/.local/lib/python3.9/site-packages/dask_jobqueue/core.py:19: FutureWarning: format_bytes is deprecated and will be removed in a future release. Please use dask.utils.format_bytes instead.
from distributed.utils import format_bytes, parse_bytes, tmpfile
/glade/u/home/mgrover/.local/lib/python3.9/site-packages/dask_jobqueue/core.py:19: FutureWarning: parse_bytes is deprecated and will be removed in a future release. Please use dask.utils.parse_bytes instead.
from distributed.utils import format_bytes, parse_bytes, tmpfile
/glade/u/home/mgrover/.local/lib/python3.9/site-packages/dask_jobqueue/htcondor.py:6: FutureWarning: parse_bytes is deprecated and will be removed in a future release. Please use dask.utils.parse_bytes instead.
from distributed.utils import parse_bytes
/glade/work/mgrover/miniconda3/envs/cesm-collections-dev/lib/python3.9/site-packages/distributed/node.py:160: UserWarning: Port 8787 is already in use.
Perhaps you already have a cluster running?
Hosting the HTTP server on port 39863 instead
warnings.warn(
client
Client
Client-dfbd23e7-0142-11ec-99a0-3cecef1b11fa
| Connection method: Cluster object | Cluster type: PBSCluster |
| Dashboard: https://jupyterhub.hpc.ucar.edu/stable/user/mgrover/proxy/39863/status |
Cluster Info
PBSCluster
23b2e2ce
| Dashboard: https://jupyterhub.hpc.ucar.edu/stable/user/mgrover/proxy/39863/status | Workers: 0 |
| Total threads: 0 | Total memory: 0 B |
Scheduler Info
Scheduler
Scheduler-a2b814ae-8f95-41a4-94e5-3a71e5f1cbd6
| Comm: tcp://10.12.206.54:39755 | Workers: 0 |
| Dashboard: https://jupyterhub.hpc.ucar.edu/stable/user/mgrover/proxy/39863/status | Total threads: 0 |
| Started: Just now | Total memory: 0 B |
Workers
col = intake.open_esm_datastore(
"/glade/work/mgrover/cesm-validation-catalog.json",
csv_kwargs={"converters": {"variables": ast.literal_eval}},
sep="/",
)
col
None catalog with 8 dataset(s) from 7402 asset(s):
| unique | |
|---|---|
| component | 1 |
| stream | 4 |
| date | 2501 |
| case | 2 |
| member_id | 1 |
| frequency | 4 |
| variables | 545 |
| path | 7402 |
cat = col.search(
variables='TEMP',
stream='pop.h',
)
cat
None catalog with 2 dataset(s) from 2400 asset(s):
| unique | |
|---|---|
| component | 1 |
| stream | 1 |
| date | 1200 |
| case | 2 |
| member_id | 1 |
| frequency | 1 |
| variables | 434 |
| path | 2400 |
sub = cat.search(date=sorted(cat.df.date.unique())[:12])
dsets = sub.to_dataset_dict(cdf_kwargs={'use_cftime': True, 'chunks': {'time': 10}})
--> The keys in the returned dictionary of datasets are constructed as follows:
'component/stream/case'
100.00% [2/2 00:00<00:00]
Do the actual plottingΒΆ
import xarray as xr
from cartopy import crs
import holoviews as hv
import holoviews as hv
import geoviews as gv
import datashader
from holoviews.operation.datashader import regrid, shade, datashade, rasterize
import matplotlib.pyplot as plt
hv.extension('bokeh')
import panel as pn
pn.extension()
def plot_interactive(ds, variable, cmap='magma'):
"""Plots an interactive plot using holoviews"""
da = ds[variable]
da['nlon'] = ds.nlon
da['nlat'] = ds.nlat
da = da.drop_vars(['TLAT', 'TLONG', 'ULAT', 'ULONG'])
hvds = hv.Dataset(da, kdims=['time', 'z_t', 'nlon', 'nlat'])
quadmesh = hvds.to(hv.QuadMesh, kdims=["nlon", "nlat"],dynamic=True)
datashade_obj = rasterize(quadmesh, precompute=True, cmap=plt.cm.RdBu_r).opts(colorbar=True, width=600, height=400, cmap=cmap, tools=['hover'])
return datashade_obj.relabel(f'{ds.title}')
def plot_comparison(ds1, ds2, variable):
diff = ds1[[variable]] - ds2[[variable]]
diff.attrs['title'] = f'{ds1.title} - {ds2.title}'
diff[variable].attrs['units'] = ds1[variable].units
plots = hv.Layout([plot_interactive(ds1, variable) + plot_interactive(ds2, variable) + plot_interactive(diff, variable, cmap='seismic')]).cols(2)
panel = pn.Column(pn.pane.Markdown(f"## {ds1[variable].name} {ds1[variable].units}", align="center"), plots)
return panel
ds = dsets['ocn/pop.h/b1850.f19_g17.validation_nuopc.004']
ds2 = dsets['ocn/pop.h/b1850.f19_g17.validation_mct.004']
plot_comparison(ds, ds2, 'TEMP')